package gbench.appdemo.crawler;

import org.jsoup.Jsoup;
import org.junit.jupiter.api.Test;
import static gbench.common.client.AbstractHttpClient.*;
import gbench.common.fs.FileSystem;
import gbench.commonApp.data.tushare.TuShareApi;
import static gbench.common.tree.LittleTree.IRecord.*;
import java.text.MessageFormat;
import java.util.concurrent.CountDownLatch;
import java.util.regex.Pattern;
import java.util.stream.Stream;

/**
 * 
 * @author gbench
 *
 */
public class MaoXuan {
    
    /**
     * 
     * @param url
     * @param book
     */
    final void download(String url,String book) {
        final var api = new TuShareApi(url,"");
        final var latch = new CountDownLatch(1);
        api.ajax(REC("$method","get","$success_type","HR2U",
            "$success", (HR2U<String>)response -> {
                final var body_pattern = Pattern.compile("<div class=\"main1\">(.+)</div>");
                final var myurl = Pattern.compile(".*/([^/]+)$").matcher(url);
                myurl.find();
                final var id = myurl.group(1);
                try {
                    final String page = str(response);// 获取网页信息
                    final var p1 = Pattern.compile("<h1 id=\"myheader\">(.+)</h1>").matcher(page);
                    p1.find();
                    final var title = Jsoup.parse(p1.group(1)).text().replace("*", "");
                    final var body_matcher = body_pattern.matcher(page);
                    if(body_matcher.find()) {
                        //final var html = body_matcher.group(1);// 提取文档
                        //final var doc = Jsoup.parse(html);
                        //写入html文件
                        FileSystem.write(MessageFormat.format("{0}/{1}/html/{2}_{3}.html", this.home,book,id,title),page);
                    }
                } catch (Exception e) {
                    e.printStackTrace();
                }
                latch.countDown();
             return null;
        }));
        try {
            latch.await();
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
    
    public String home = 
            //"D:\\sliced\\ebook\\texts";
          "D:\\sliced\\ebook\\maoxuan";
    
    /**
     * 史记
     */
    @Test
    public void shiji() {
        Stream.iterate(1, i->i<=368,i->i+1)
        .map(i->MessageFormat.format("http://www.dodobook.com/index.php?id=books/maozedongxuanji/{0}",i))
        .forEach(url->download(url,"毛选"));
    }
   

}
